{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|default_exp models.TabFusionTransformer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TabFusionTransformer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a a Pytorch implementeation of  TabTransformerTransformer created by Ignacio Oguiza (oguiza@timeseriesAI.co)\n",
    "\n",
    "This implementation is inspired by:\n",
    "\n",
    "Huang, X., Khetan, A., Cvitkovic, M., & Karnin, Z. (2020). <span style=\"color:dodgerblue\">**TabTransformer: Tabular Data Modeling Using Contextual Embeddings**</span>. arXiv preprint https://arxiv.org/pdf/2012.06678\n",
    "\n",
    "Official repo: https://github.com/awslabs/autogluon/tree/master/tabular/src/autogluon/tabular/models/tab_transformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|eval: false\n",
    "#|hide\n",
    "from tsai.imports import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "# This is a modified Pytorch implementation based on TabTransformer created by Ignacio Oguiza (oguiza@timeseriesAI.co):\n",
    "# Huang, X., Khetan, A., Cvitkovic, M., & Karnin, Z. (2020). \n",
    "# TabTransformer: Tabular Data Modeling Using Contextual Embeddings. \n",
    "# arXiv preprint https://arxiv.org/pdf/2012.06678\n",
    "# Official repo: https://github.com/awslabs/autogluon/tree/master/tabular/src/autogluon/tabular/models/tab_transformer\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from collections import OrderedDict\n",
    "\n",
    "\n",
    "def ifnone(a, b):\n",
    "    # From fastai.fastcore\n",
    "    \"`b` if `a` is None else `a`\"\n",
    "    return b if a is None else a\n",
    "\n",
    "\n",
    "class _Flatten(nn.Module):\n",
    "    def __init__(self, full=False):\n",
    "        super().__init__()\n",
    "        self.full = full\n",
    "    def forward(self, x):\n",
    "        return x.view(-1) if self.full else x.view(x.size(0), -1)\n",
    "\n",
    "\n",
    "class Sequential(nn.Sequential):\n",
    "    \"\"\"Class that allows you to pass one or multiple inputs\"\"\"\n",
    "    def forward(self, *x):\n",
    "        for i, module in enumerate(self._modules.values()):\n",
    "            x = module(*x) if isinstance(x, (list, tuple)) else module(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "class _MLP(nn.Module):\n",
    "    def __init__(self, dims, bn=False, act=None, skip=False, dropout=0., bn_final=False):\n",
    "        super().__init__()\n",
    "        dims_pairs = list(zip(dims[:-1], dims[1:]))\n",
    "        layers = []\n",
    "        for i, (dim_in, dim_out) in enumerate(dims_pairs):\n",
    "            is_last = i >= (len(dims) - 2)\n",
    "            if bn and (not is_last or bn_final): layers.append(nn.BatchNorm1d(dim_in))\n",
    "            if dropout and not is_last:\n",
    "                layers.append(nn.Dropout(dropout))\n",
    "            layers.append(nn.Linear(dim_in, dim_out))\n",
    "            if is_last: break\n",
    "            layers.append(ifnone(act, nn.ReLU()))\n",
    "        self.mlp = nn.Sequential(*layers)\n",
    "        self.shortcut = nn.Linear(dims[0], dims[-1]) if skip else None\n",
    "\n",
    "    def forward(self, x):\n",
    "        if self.shortcut is not None: \n",
    "            return self.mlp(x) + self.shortcut(x)\n",
    "        else:\n",
    "            return self.mlp(x)\n",
    "        \n",
    "\n",
    "class _ScaledDotProductAttention(nn.Module):\n",
    "    def __init__(self, d_k:int, res_attention:bool=False): \n",
    "        super().__init__()\n",
    "        self.d_k,self.res_attention = d_k,res_attention\n",
    "        \n",
    "    def forward(self, q, k, v, prev=None, attn_mask=None):\n",
    "\n",
    "        # MatMul (q, k) - similarity scores for all pairs of positions in an input sequence\n",
    "        scores = torch.matmul(q, k)                                    # scores : [bs x n_heads x q_len x q_len]\n",
    "\n",
    "        # Scale\n",
    "        scores = scores / (self.d_k ** 0.5)\n",
    "\n",
    "        # Attention mask (optional)\n",
    "        if attn_mask is not None:                                     # mask with shape [q_len x q_len]\n",
    "            if attn_mask.dtype == torch.bool:\n",
    "                scores.masked_fill_(attn_mask, float('-inf'))\n",
    "            else:\n",
    "                scores += attn_mask\n",
    "\n",
    "        # SoftMax\n",
    "        if prev is not None: scores = scores + prev\n",
    "\n",
    "        attn = F.softmax(scores, dim=-1)                               # attn   : [bs x n_heads x q_len x q_len]\n",
    "\n",
    "        # MatMul (attn, v)\n",
    "        context = torch.matmul(attn, v)                                # context: [bs x n_heads x q_len x d_v]\n",
    "\n",
    "        if self.res_attention: return context, attn, scores\n",
    "        else: return context, attn\n",
    "\n",
    "\n",
    "class _MultiheadAttention(nn.Module):\n",
    "    def __init__(self, d_model:int, n_heads:int, d_k:int, d_v:int, res_attention:bool=False):\n",
    "        \"\"\"Input shape:  Q, K, V:[batch_size (bs) x q_len x d_model], mask:[q_len x q_len]\"\"\"\n",
    "        super().__init__()\n",
    "        self.n_heads, self.d_k, self.d_v = n_heads, d_k, d_v\n",
    "\n",
    "        self.W_Q = nn.Linear(d_model, d_k * n_heads, bias=False)\n",
    "        self.W_K = nn.Linear(d_model, d_k * n_heads, bias=False)\n",
    "        self.W_V = nn.Linear(d_model, d_v * n_heads, bias=False)\n",
    "\n",
    "        self.W_O = nn.Linear(n_heads * d_v, d_model, bias=False)\n",
    "\n",
    "        self.res_attention = res_attention\n",
    "\n",
    "        # Scaled Dot-Product Attention (multiple heads)\n",
    "        if self.res_attention:\n",
    "            self.sdp_attn = _ScaledDotProductAttention(self.d_k, self.res_attention)\n",
    "        else:\n",
    "            self.sdp_attn = _ScaledDotProductAttention(self.d_k)\n",
    "\n",
    "        \n",
    "    def forward(self, Q, K, V, prev=None, attn_mask=None):\n",
    "\n",
    "        bs = Q.size(0)\n",
    "\n",
    "        # Linear (+ split in multiple heads)\n",
    "        q_s = self.W_Q(Q).view(bs, -1, self.n_heads, self.d_k).transpose(1,2)       # q_s    : [bs x n_heads x q_len x d_k]\n",
    "        k_s = self.W_K(K).view(bs, -1, self.n_heads, self.d_k).permute(0,2,3,1)     # k_s    : [bs x n_heads x d_k x q_len] - transpose(1,2) + transpose(2,3)\n",
    "        v_s = self.W_V(V).view(bs, -1, self.n_heads, self.d_v).transpose(1,2)       # v_s    : [bs x n_heads x q_len x d_v]\n",
    "\n",
    "        # Scaled Dot-Product Attention (multiple heads)\n",
    "        if self.res_attention:\n",
    "            context, attn, scores = self.sdp_attn(q_s, k_s, v_s, prev=prev, attn_mask=attn_mask)\n",
    "        else:\n",
    "            context, attn = self.sdp_attn(q_s, k_s, v_s, attn_mask=attn_mask)\n",
    "        # context: [bs x n_heads x q_len x d_v], attn: [bs x n_heads x q_len x q_len]\n",
    "\n",
    "        # Concat\n",
    "        context = context.transpose(1, 2).contiguous().view(bs, -1, self.n_heads * self.d_v) # context: [bs x q_len x n_heads * d_v]\n",
    "\n",
    "        # Linear\n",
    "        output = self.W_O(context)                                                           # context: [bs x q_len x d_model]\n",
    "\n",
    "        if self.res_attention: return output, attn, scores\n",
    "        else: return output, attn                                                            # output: [bs x q_len x d_model]\n",
    "\n",
    "\n",
    "class _TabFusionEncoderLayer(nn.Module):\n",
    "    def __init__(self, q_len, d_model, n_heads, d_k=None, d_v=None, d_ff=None, \n",
    "                 res_dropout=0.1, activation=\"gelu\", res_attention=False):\n",
    "\n",
    "        super().__init__()\n",
    "        assert not d_model%n_heads, f\"d_model ({d_model}) must be divisible by n_heads ({n_heads})\"\n",
    "        d_k = ifnone(d_k, d_model // n_heads)\n",
    "        d_v = ifnone(d_v, d_model // n_heads)\n",
    "        d_ff = ifnone(d_ff, d_model * 4)\n",
    "\n",
    "        # Multi-Head attention\n",
    "        self.res_attention = res_attention\n",
    "        self.self_attn = _MultiheadAttention(d_model, n_heads, d_k, d_v, res_attention=res_attention)\n",
    "\n",
    "        # Add & Norm\n",
    "        self.dropout_attn = nn.Dropout(res_dropout)\n",
    "        self.layernorm_attn = nn.LayerNorm(d_model)\n",
    "\n",
    "        # Position-wise Feed-Forward\n",
    "        self.ff = nn.Sequential(nn.Linear(d_model, d_ff), self._get_activation_fn(activation), nn.Linear(d_ff, d_model))\n",
    "\n",
    "        # Add & Norm\n",
    "        self.dropout_ffn = nn.Dropout(res_dropout)\n",
    "        self.layernorm_ffn = nn.LayerNorm(d_model)\n",
    "\n",
    "    def forward(self, src, prev=None, attn_mask=None):\n",
    "\n",
    "        # Multi-Head attention sublayer\n",
    "        ## Multi-Head attention\n",
    "        if self.res_attention:\n",
    "            src2, attn, scores = self.self_attn(src, src, src, prev, attn_mask=attn_mask)\n",
    "        else:\n",
    "            src2, attn = self.self_attn(src, src, src, attn_mask=attn_mask)\n",
    "        self.attn = attn\n",
    "        ## Add & Norm\n",
    "        src = src + self.dropout_attn(src2) # Add: residual connection with residual dropout\n",
    "        src = self.layernorm_attn(src) # Norm: layernorm \n",
    "\n",
    "        # Feed-forward sublayer\n",
    "        ## Position-wise Feed-Forward\n",
    "        src2 = self.ff(src)\n",
    "        ## Add & Norm\n",
    "        src = src + self.dropout_ffn(src2) # Add: residual connection with residual dropout\n",
    "        src = self.layernorm_ffn(src) # Norm: layernorm\n",
    "\n",
    "        if self.res_attention:\n",
    "            return src, scores\n",
    "        else:\n",
    "            return src\n",
    "\n",
    "    def _get_activation_fn(self, activation):\n",
    "        if callable(activation): return activation()\n",
    "        elif activation.lower() == \"relu\": return nn.ReLU()\n",
    "        elif activation.lower() == \"gelu\": return nn.GELU()\n",
    "        elif activation.lower() == \"mish\": return Mish()\n",
    "        raise ValueError(f'{activation} is not available. You can use \"relu\", \"gelu\", \"mish\" or a callable')\n",
    "\n",
    "\n",
    "class _TabFusionEncoder(nn.Module):\n",
    "    def __init__(self, q_len, d_model, n_heads, d_k=None, d_v=None, d_ff=None, res_dropout=0.1, activation='gelu', res_attention=False, n_layers=1):\n",
    "        super().__init__()\n",
    "        self.layers = nn.ModuleList([_TabFusionEncoderLayer(q_len, d_model, n_heads=n_heads, d_k=d_k, d_v=d_v, d_ff=d_ff, res_dropout=res_dropout, \n",
    "                                                            activation=activation, res_attention=res_attention) for i in range(n_layers)])\n",
    "        self.res_attention = res_attention\n",
    "\n",
    "    def forward(self, src, attn_mask=None):\n",
    "        output = src\n",
    "        scores = None\n",
    "        if self.res_attention:\n",
    "            for mod in self.layers: output, scores = mod(output, prev=scores, attn_mask=attn_mask)\n",
    "            return output\n",
    "        else:\n",
    "            for mod in self.layers: output = mod(output, attn_mask=attn_mask)\n",
    "            return output\n",
    "\n",
    "\n",
    "class TabFusionBackbone(nn.Module):\n",
    "    def __init__(self, classes, cont_names, d_model=32, n_layers=6, n_heads=8, d_k=None, d_v=None, d_ff=None, init=True,\n",
    "                 res_attention=True, attention_act='gelu', res_dropout=0.):\n",
    "\n",
    "        super().__init__()\n",
    "        \n",
    "        # Categorical\n",
    "        n_cat = len(classes)\n",
    "        n_classes = [len(v) for v in classes.values()]\n",
    "        self.n_emb = sum(n_classes)\n",
    "        self.embeds = nn.ModuleList([nn.Embedding(ni, d_model) for ni in n_classes])\n",
    "        \n",
    "        # Continuous\n",
    "        n_cont = len(cont_names)\n",
    "        self.n_cont = n_cont\n",
    "        self.conv = nn.Conv1d(1, d_model, 1)\n",
    "        if init: nn.init.kaiming_normal_(self.conv.weight)\n",
    "\n",
    "        # Transformer\n",
    "        self.res_drop = nn.Dropout(res_dropout) if res_dropout else None\n",
    "        self.pos_enc = nn.Parameter(torch.zeros(1, (n_cat  + n_cont), d_model))\n",
    "        self.transformer = _TabFusionEncoder(n_cat + n_cont, d_model, n_heads=n_heads, d_k=d_k, d_v=d_v, d_ff=d_ff, res_dropout=res_dropout, \n",
    "                                             activation=attention_act, res_attention=res_attention, n_layers=n_layers)\n",
    "\n",
    "\n",
    "    def forward(self, x_cat, x_cont=None):\n",
    "        \n",
    "        # Input encoding\n",
    "        if self.n_emb != 0:\n",
    "            x = [e(x_cat[:,i]).unsqueeze(1) for i,e in enumerate(self.embeds)]\n",
    "            x = torch.cat(x, 1)\n",
    "        if self.n_cont != 0:\n",
    "            x_cont = self.conv(x_cont.unsqueeze(1)).transpose(1,2)\n",
    "            x = torch.cat([x, x_cont], 1) if self.n_emb != 0 else x_cont\n",
    "\n",
    "        # Transformer\n",
    "        x += self.pos_enc\n",
    "        if self.res_drop is not None: x = self.res_drop(x)\n",
    "        x = self.transformer(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "\n",
    "class TabFusionTransformer(Sequential):\n",
    "    def __init__(self, classes, cont_names, c_out, \n",
    "                 d_model=32, n_layers=6, n_heads=8, d_k=None, d_v=None, d_ff=None, res_attention=True, attention_act='gelu', res_dropout=0.,\n",
    "                 fc_mults=(4, 2), fc_dropout=0., fc_act=None, fc_skip=False, fc_bn=False, bn_final=False, init=True):\n",
    "\n",
    "        super().__init__()\n",
    "        \n",
    "        # Backbone\n",
    "        backbone = TabFusionBackbone(classes, cont_names, d_model=d_model, n_layers=n_layers, n_heads=n_heads, d_k=d_k, d_v=d_v, d_ff=d_ff, init=init,\n",
    "                                     res_attention=res_attention, attention_act=attention_act, res_dropout=res_dropout)\n",
    "        \n",
    "        # Head\n",
    "        mlp_input_size = (d_model * (len(classes)  + len(cont_names)))\n",
    "        hidden_dimensions = list(map(lambda t: int(mlp_input_size * t), fc_mults))\n",
    "        all_dimensions = [mlp_input_size, *hidden_dimensions, c_out]\n",
    "        self.head_nf = mlp_input_size\n",
    "        head = nn.Sequential(*[_Flatten(), _MLP(all_dimensions, act=fc_act, skip=fc_skip, bn=fc_bn, dropout=fc_dropout, bn_final=bn_final)])\n",
    "\n",
    "        super().__init__(OrderedDict([('backbone', backbone), ('head', head)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.tabular.all import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = untar_data(URLs.ADULT_SAMPLE)\n",
    "df = pd.read_csv(path/'adult.csv')\n",
    "dls = TabularDataLoaders.from_csv(path/'adult.csv', path=path, y_names=\"salary\",\n",
    "    cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race'],\n",
    "    cont_names = ['age', 'fnlwgt', 'education-num'],\n",
    "    procs = [Categorify, FillMissing, Normalize])\n",
    "x_cat, x_cont, yb = first(dls.train)\n",
    "model = TabFusionTransformer(dls.classes, dls.cont_names, dls.c)\n",
    "test_eq(model(x_cat, x_cont).shape, (dls.train.bs, dls.c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "class TSTabFusionTransformer(nn.Module):\n",
    "    def __init__(self, c_in, c_out, seq_len, classes, cont_names, \n",
    "                 d_model=32, n_layers=6, n_heads=8, d_k=None, d_v=None, d_ff=None, res_attention=True, attention_act='gelu', res_dropout=0., \n",
    "                 fc_mults=(1, .5), fc_dropout=0., fc_act=None, fc_skip=False, fc_bn=False, bn_final=False, init=True):\n",
    "\n",
    "        super().__init__()\n",
    "        \n",
    "        # Time series\n",
    "        self.W_P = nn.Conv1d(c_in, d_model, 1)\n",
    "        \n",
    "        # Categorical\n",
    "        n_cat = len(classes)\n",
    "        n_classes = [len(v) for v in classes.values()]\n",
    "        self.n_emb = sum(n_classes)\n",
    "        self.embeds = nn.ModuleList([nn.Embedding(ni, d_model) for ni in n_classes])\n",
    "        \n",
    "        # Continuous\n",
    "        n_cont = len(cont_names)\n",
    "        self.n_cont = n_cont\n",
    "        self.conv = nn.Conv1d(1, d_model, 1)\n",
    "        if init: nn.init.kaiming_normal_(self.conv.weight)\n",
    "\n",
    "        # Transformer\n",
    "        self.res_drop = nn.Dropout(res_dropout) if res_dropout else None\n",
    "        self.pos_enc = nn.Parameter(torch.zeros(1, (n_cat  + n_cont + seq_len), d_model))\n",
    "        self.transformer = _TabFusionEncoder(n_cat + n_cont, d_model, n_heads=n_heads, d_k=d_k, d_v=d_v, d_ff=d_ff, res_dropout=res_dropout, \n",
    "                                             activation=attention_act, res_attention=res_attention, n_layers=n_layers)\n",
    "        \n",
    "        # Head\n",
    "        mlp_input_size = (d_model * (n_cat  + n_cont + seq_len))\n",
    "        hidden_dimensions = list(map(lambda t: int(mlp_input_size * t), fc_mults))\n",
    "        all_dimensions = [mlp_input_size, *hidden_dimensions, c_out]\n",
    "        self.head_nf = mlp_input_size\n",
    "        self.head = nn.Sequential(*[_Flatten(), _MLP(all_dimensions, act=fc_act, skip=fc_skip, bn=fc_bn, dropout=fc_dropout, bn_final=bn_final)])\n",
    "\n",
    "    def forward(self, x):\n",
    "        x_ts, (x_cat, x_cont) = x\n",
    "        \n",
    "        # Time series\n",
    "        x = self.W_P(x_ts).transpose(1,2)\n",
    "        \n",
    "        # Input encoding\n",
    "        if self.n_emb != 0:\n",
    "            x_cat = [e(x_cat[:,i]).unsqueeze(1) for i,e in enumerate(self.embeds)]\n",
    "            x_cat = torch.cat(x_cat, 1)\n",
    "            x = torch.cat([x, x_cat], 1)\n",
    "        if self.n_cont != 0:\n",
    "            x_cont = self.conv(x_cont.unsqueeze(1)).transpose(1,2)\n",
    "            x = torch.cat([x, x_cont], 1)\n",
    "\n",
    "        # Transformer\n",
    "        x += self.pos_enc\n",
    "        if self.res_drop is not None: x = self.res_drop(x)\n",
    "        x = self.transformer(x)\n",
    "\n",
    "        # Head\n",
    "        x = self.head(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = {'education': ['#na#', '10th', '11th', '12th', '1st-4th', '5th-6th', '7th-8th', '9th', 'Assoc-acdm', 'Assoc-voc', 'Bachelors', 'Doctorate', \n",
    "                         'HS-grad', 'Masters', 'Preschool', 'Prof-school', 'Some-college'],\n",
    " 'education-num_na': ['#na#', False, True],\n",
    " 'marital-status': ['#na#', 'Divorced', 'Married-AF-spouse', 'Married-civ-spouse', 'Married-spouse-absent', 'Never-married', 'Separated', 'Widowed'],\n",
    " 'occupation': ['#na#', '?', 'Adm-clerical', 'Armed-Forces', 'Craft-repair', 'Exec-managerial', 'Farming-fishing', 'Handlers-cleaners', 'Machine-op-inspct', \n",
    "                'Other-service', 'Priv-house-serv', 'Prof-specialty', 'Protective-serv', 'Sales', 'Tech-support', 'Transport-moving'],\n",
    " 'race': ['#na#', 'Amer-Indian-Eskimo', 'Asian-Pac-Islander', 'Black', 'Other', 'White'],\n",
    " 'relationship': ['#na#', 'Husband', 'Not-in-family', 'Other-relative', 'Own-child', 'Unmarried', 'Wife'],\n",
    " 'workclass': ['#na#', '?', 'Federal-gov', 'Local-gov', 'Never-worked', 'Private', 'Self-emp-inc', 'Self-emp-not-inc', 'State-gov', 'Without-pay']}\n",
    "\n",
    "cont_names = ['a', 'b', 'c']\n",
    "c_out = 6\n",
    "x_ts = torch.randn(64, 3, 10)\n",
    "x_cat = torch.randint(0,3,(64,7))\n",
    "x_cont = torch.randn(64,3)\n",
    "model = TSTabFusionTransformer(x_ts.shape[1], c_out, x_ts.shape[-1], classes, cont_names)\n",
    "x = (x_ts, (x_cat, x_cont))\n",
    "test_eq(model(x).shape, (x_ts.shape[0], c_out))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": "IPython.notebook.save_checkpoint();",
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/nacho/notebooks/tsai/nbs/122_models.TabFusionTransformer.ipynb saved at 2022-11-09 13:12:15\n",
      "Correct notebook to script conversion! 😃\n",
      "Wednesday 09/11/22 13:12:18 CET\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <audio  controls=\"controls\" autoplay=\"autoplay\">\n",
       "                    <source src=\"data:audio/wav;base64,UklGRvQHAABXQVZFZm10IBAAAAABAAEAECcAACBOAAACABAAZGF0YdAHAAAAAPF/iPh/gOoOon6w6ayCoR2ZeyfbjobxK+F2Hs0XjKc5i3DGvzaTlEaraE+zz5uLUl9f46fHpWJdxVSrnfmw8mYEScqUP70cb0Q8X41uysJ1si6Eh1jYzXp9IE2DzOYsftYRyoCY9dJ/8QICgIcEun8D9PmAaBPlfT7lq4MFIlh61tYPiCswIHX+yBaOqT1QbuW7qpVQSv9lu6+xnvRVSlyopAypbGBTUdSalrSTaUBFYpInwUpxOzhti5TOdndyKhCGrdwAfBUcXIJB69p+Vw1egB76+n9q/h6ADglbf4LvnIHfF/981ODThF4m8HiS0riJVjQ6c+/EOZCYQfJrGrhBmPVNMmNArLKhQlkXWYqhbaxXY8ZNHphLuBJsZUEckCTFVHMgNKGJytIDeSUmw4QN4Qx9pReTgb3vYX/TCBuApf75f+P5Y4CRDdN+B+tngk8c8nt03CKGqipgd13OhotwOC5x9MCAknFFcmlmtPmagFFFYOCo0qRzXMhVi57pryNmIEqJlRi8bm52PfuNM8k4dfQv+4cO12l6zCGdg3jl730uE/KAPvS+f0wEAoAsA89/XfXQgBESIn6S5luDtiC8eh/YmIfpLqt1OMp5jXg8/24MveqUNUnPZsqw0Z3yVDldnaUOqIZfXlKrm36zzWhjRhaT+r+ncHI5/otUzfd2uSt7hl/bqXtoHaCC6+mqfrAOeoDD+PJ/xf8RgLMHfH/b8GeBihZIfSXidoQSJWB52NM1iRkzz3MkxpKPbUCrbDu5d5fgTAxkSK3JoEhYD1p2omere2LZTuqYLbdWa49Cx5Dww7tyXDUnioXRkHhwJyKFvd/AfPoYy4Fl7j1/LQorgEr9/X89+0qAOAwAf13sJoL8Gkd8wt25hWIp3Heez/eKODfPcSPCzpFNRDVqf7UlmnNQKGHgqd+jgVvJVm2f265QZTpLS5byur1tpT6ajvrHq3Q2MXWIxtUCehoj8YMk5LB9hRQegeTypn+nBQWA0QHgf7f2q4C5EFt+5ucOg2YfHXtq2SSHpS0ydnTL4IxFO6pvNb4ulBdInWfcsfSc7VMmXpSmE6eeXmZThJxpsgRohEfOk86+AHCoOpOMFsx1dv8s6oYT2k17uR7ngpXod34IEJqAaPfnfyABCIBZBpl/NPI2gTQVjX134x2ExSPMeR7VtYjZMWJ0W8ftjkA/YW1durCWykvjZFKu4p9LVwVbZKNkqpxh6U+6mRC2mGq2Q3SRvsIgcpc2sIpD0Bp4uiiFhW3ecXxOGgaCDe0Vf4cLPoDv+/5/mfw1gN4KKX+17emBqBmYfBHfVYUZKFR44NBtiv41bHJUwx+RJkP1apu2VJlkTwli4qrwoo1ax1dToNCtemRSTBGXz7kJbdM/PY/Dxht0dTLziH7Ul3loJEiE0uJsfdsVTYGL8Yt/AgcMgHYA7X8S+IqAYA+QfjzpxIIVHnp7tdqzhmAstXaxzEqMETpScGC/dJP3Rmdo8LIZnOVSEF+Opxumsl1sVF+dVrE5Z6NIiZSkvVdv2zsqjdnK8HVDLlyHyNjuegogM4NA5z9+YRG9gA722H97AgOA/gSyf43zCIHdE899yuTIg3ciNXpm1jmImTDwdJPITI4RPhRugbvslbFKt2Vfr/6eTFb4W1WkY6m6YPdQjJr2tNZp3EQlko7BgXHRNz2LAc+gdwMq7IUf3R58ohtFgrbr6n7hDFWAlPr8f/T9I4CECU9/De+vgVQY5nxh4POEzybJeCTS5YnCNAZzhsRzkP1Bsmu4t4aYU07nYuerA6KWWcJYO6HHrKJjaE3Zl624UWz/QOOPjcWHc7QzdIk40yl5tCWjhIDhJX0xF4CBMvBsf10IF4Ac//Z/bPlsgAcOwn6S6n6CwxzUewLcRoYaKzV38M23i9o493CNwL6S1UUuaQe0QpvbUfdfiqglpcRccFU+nkWwambASUiVfLyqbg49xY2eyWh1hy/Sh37XjHpaIYKD7OUEfrgS5IC09MV/1gMBgKMDyH/n9N6AhhINfh7mdoMoIZt6r9fAh1cvfHXNya6N4DzDbqi8K5WWSYlmbbAdnkpV6FxJpWSo1V8DUmGb3rMRaQBG2JJgwN9wCDnNi8HNI3dKK1aG0dvHe/UciIJf6rt+Og5wgDn59X9P/xWAKQhxf2XweYH+FjB9suGVhIMlOnlo02GJhTOdc7vFyo/TQGxs2Li7lz9NwmPurBihnVi7WSWiwKvGYntOpJiOt5drKUKMkFnE8HLxNPmJ9NG4eP8mAYUv4Np8hhi3gdruSX+3CSWAwP38f8f6UoCuDPF+6Os8gnAbKnxQ3d2F0imydzDPKIuiN5lxu8EKkrFE82kftW2az1DbYImpMqTUW3FWIJ83r5hl2koJlla7+m0+PmSOZcjcdMgwS4g11iZ6qCLUg5jkxn0QFA6BWvOvfzEFBIBHAtp/Qfa3gC4RSH5y5yeD2B/8evnYS4cULgR2CMsUja47cG/QvW6UeEhXZ3+xP51GVNVdP6Zpp+1eDFM5nMeySWghR4+TNL85cD46YIyCzKJ2kCzEhoTabXtGHs+CCemJfpMPjoDe9+t/qQALgM8Gj3++8UaBqRV2fQTjO4Q3JKd5r9TgiEYyMHTxxiWPpz8jbfq585YpTJpk960xoKFXsVoTo7yq6GGMTw==\" type=\"audio/wav\" />\n",
       "                    Your browser does not support the audio element.\n",
       "                </audio>\n",
       "              "
      ],
      "text/plain": [
       "<IPython.lib.display.Audio object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#|eval: false\n",
    "#|hide\n",
    "from tsai.export import get_nb_name; nb_name = get_nb_name(locals())\n",
    "from tsai.imports import create_scripts; create_scripts(nb_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
